Benchmark#

Whenever we make changes to the code we will rerun a simple benchmark in order to keep track of how the simulation results might change over time. It is in general hard to verify that the equations are implemented correctly because there exist no analytic solutions to the equations. However, by running the same benchmark over time we can see if the changes we make will impact the simulation results. Here the different biomarkers are taken from traces evaluated a the center of at slab that is simulated for 1000 ms (see the benchmark in the library for more details)

from pathlib import Path
import json
import matplotlib.pyplot as plt
import numpy as np
from collections import defaultdict
import pandas as pd
import plotly.graph_objects as go
from plotly.subplots import make_subplots

pd.options.display.max_columns = 5

benchmark_folder = Path(".").absolute().parent / "benchmarks"

all_hashes = [f.name for f in benchmark_folder.iterdir()]

print(all_hashes)

['60e69df', 'cc84512', '3398fd2', '376167c', '12ad5f2', '322191d', 'e36932a', '0e0091e', '69503d0', '9b246fe', '5d5bd73', 'de72206', '42363b0', '67aba10', '813ba9c', 'cb33e06', '6dde0c0', 'ce433ea', 'd44db58', 'f63afc9', '172470c', '4cc3e7a', 'f4b9b88', '93c349e', 'f214f80', 'dbbec69', 'de8a7ff', '0d80766', '695cd7b', '32e43ff', 'e616f0a', '80a11fc', '2fe1309', 'cbc3038', 'c9b3528', '7ffac90', '8b11e8d', '68ae7bb', '0221cc5', '9f2fd38', '47edd23', 'd5245d4', '10592e9', '3ed0bc0', '3a87bf1', 'a4ea024', '1f7912f', '1353445', '999fcfa', '5cc1a44', '90b65d3', '5a86d28', 'edc973c', '739dc1c', '630ed09', 'c242e33', 'fbfcf55', '7f51a50', '5edda13', '1ed82dc', '71a7b0e', '947b107', '6e55904', '68926df', '5992724', 'f73b25e', '5282d4f', 'e8e6d35', '37f4c61', '4d3a653', '37fb7d1', 'f38605c', 'cf7cc9d', 'f5511ff', '3b1d61b', '09a7e5b', '0c195ff', 'e99e183', 'df9ad53', 'f702955', 'a3a23b2', 'c7df02b', '97e958d', '04a9790', '159c6bc', '9d5c1b1', 'ac17a41', 'e3254b0', '9d72060', '5580bef', 'bc47c96', '38ad52c', '7d2252c', '0648726']

You can checkout a specific commit by going to the link

ComputationalPhysiology/simcardems

where you swap out GIT_COMMIT_HASH with the specific hash, i.e

for hash in all_hashes:
    print(f"https://github.com/ComputationalPhysiology/simcardems/commit/{hash}")

https://github.com/ComputationalPhysiology/simcardems/commit/60e69df
https://github.com/ComputationalPhysiology/simcardems/commit/cc84512
https://github.com/ComputationalPhysiology/simcardems/commit/3398fd2
https://github.com/ComputationalPhysiology/simcardems/commit/376167c
https://github.com/ComputationalPhysiology/simcardems/commit/12ad5f2
https://github.com/ComputationalPhysiology/simcardems/commit/322191d
https://github.com/ComputationalPhysiology/simcardems/commit/e36932a
https://github.com/ComputationalPhysiology/simcardems/commit/0e0091e
https://github.com/ComputationalPhysiology/simcardems/commit/69503d0
https://github.com/ComputationalPhysiology/simcardems/commit/9b246fe
https://github.com/ComputationalPhysiology/simcardems/commit/5d5bd73
https://github.com/ComputationalPhysiology/simcardems/commit/de72206
https://github.com/ComputationalPhysiology/simcardems/commit/42363b0
https://github.com/ComputationalPhysiology/simcardems/commit/67aba10
https://github.com/ComputationalPhysiology/simcardems/commit/813ba9c
https://github.com/ComputationalPhysiology/simcardems/commit/cb33e06
https://github.com/ComputationalPhysiology/simcardems/commit/6dde0c0
https://github.com/ComputationalPhysiology/simcardems/commit/ce433ea
https://github.com/ComputationalPhysiology/simcardems/commit/d44db58
https://github.com/ComputationalPhysiology/simcardems/commit/f63afc9
https://github.com/ComputationalPhysiology/simcardems/commit/172470c
https://github.com/ComputationalPhysiology/simcardems/commit/4cc3e7a
https://github.com/ComputationalPhysiology/simcardems/commit/f4b9b88
https://github.com/ComputationalPhysiology/simcardems/commit/93c349e
https://github.com/ComputationalPhysiology/simcardems/commit/f214f80
https://github.com/ComputationalPhysiology/simcardems/commit/dbbec69
https://github.com/ComputationalPhysiology/simcardems/commit/de8a7ff
https://github.com/ComputationalPhysiology/simcardems/commit/0d80766
https://github.com/ComputationalPhysiology/simcardems/commit/695cd7b
https://github.com/ComputationalPhysiology/simcardems/commit/32e43ff
https://github.com/ComputationalPhysiology/simcardems/commit/e616f0a
https://github.com/ComputationalPhysiology/simcardems/commit/80a11fc
https://github.com/ComputationalPhysiology/simcardems/commit/2fe1309
https://github.com/ComputationalPhysiology/simcardems/commit/cbc3038
https://github.com/ComputationalPhysiology/simcardems/commit/c9b3528
https://github.com/ComputationalPhysiology/simcardems/commit/7ffac90
https://github.com/ComputationalPhysiology/simcardems/commit/8b11e8d
https://github.com/ComputationalPhysiology/simcardems/commit/68ae7bb
https://github.com/ComputationalPhysiology/simcardems/commit/0221cc5
https://github.com/ComputationalPhysiology/simcardems/commit/9f2fd38
https://github.com/ComputationalPhysiology/simcardems/commit/47edd23
https://github.com/ComputationalPhysiology/simcardems/commit/d5245d4
https://github.com/ComputationalPhysiology/simcardems/commit/10592e9
https://github.com/ComputationalPhysiology/simcardems/commit/3ed0bc0
https://github.com/ComputationalPhysiology/simcardems/commit/3a87bf1
https://github.com/ComputationalPhysiology/simcardems/commit/a4ea024
https://github.com/ComputationalPhysiology/simcardems/commit/1f7912f
https://github.com/ComputationalPhysiology/simcardems/commit/1353445
https://github.com/ComputationalPhysiology/simcardems/commit/999fcfa
https://github.com/ComputationalPhysiology/simcardems/commit/5cc1a44
https://github.com/ComputationalPhysiology/simcardems/commit/90b65d3
https://github.com/ComputationalPhysiology/simcardems/commit/5a86d28
https://github.com/ComputationalPhysiology/simcardems/commit/edc973c
https://github.com/ComputationalPhysiology/simcardems/commit/739dc1c
https://github.com/ComputationalPhysiology/simcardems/commit/630ed09
https://github.com/ComputationalPhysiology/simcardems/commit/c242e33
https://github.com/ComputationalPhysiology/simcardems/commit/fbfcf55
https://github.com/ComputationalPhysiology/simcardems/commit/7f51a50
https://github.com/ComputationalPhysiology/simcardems/commit/5edda13
https://github.com/ComputationalPhysiology/simcardems/commit/1ed82dc
https://github.com/ComputationalPhysiology/simcardems/commit/71a7b0e
https://github.com/ComputationalPhysiology/simcardems/commit/947b107
https://github.com/ComputationalPhysiology/simcardems/commit/6e55904
https://github.com/ComputationalPhysiology/simcardems/commit/68926df
https://github.com/ComputationalPhysiology/simcardems/commit/5992724
https://github.com/ComputationalPhysiology/simcardems/commit/f73b25e
https://github.com/ComputationalPhysiology/simcardems/commit/5282d4f
https://github.com/ComputationalPhysiology/simcardems/commit/e8e6d35
https://github.com/ComputationalPhysiology/simcardems/commit/37f4c61
https://github.com/ComputationalPhysiology/simcardems/commit/4d3a653
https://github.com/ComputationalPhysiology/simcardems/commit/37fb7d1
https://github.com/ComputationalPhysiology/simcardems/commit/f38605c
https://github.com/ComputationalPhysiology/simcardems/commit/cf7cc9d
https://github.com/ComputationalPhysiology/simcardems/commit/f5511ff
https://github.com/ComputationalPhysiology/simcardems/commit/3b1d61b
https://github.com/ComputationalPhysiology/simcardems/commit/09a7e5b
https://github.com/ComputationalPhysiology/simcardems/commit/0c195ff
https://github.com/ComputationalPhysiology/simcardems/commit/e99e183
https://github.com/ComputationalPhysiology/simcardems/commit/df9ad53
https://github.com/ComputationalPhysiology/simcardems/commit/f702955
https://github.com/ComputationalPhysiology/simcardems/commit/a3a23b2
https://github.com/ComputationalPhysiology/simcardems/commit/c7df02b
https://github.com/ComputationalPhysiology/simcardems/commit/97e958d
https://github.com/ComputationalPhysiology/simcardems/commit/04a9790
https://github.com/ComputationalPhysiology/simcardems/commit/159c6bc
https://github.com/ComputationalPhysiology/simcardems/commit/9d5c1b1
https://github.com/ComputationalPhysiology/simcardems/commit/ac17a41
https://github.com/ComputationalPhysiology/simcardems/commit/e3254b0
https://github.com/ComputationalPhysiology/simcardems/commit/9d72060
https://github.com/ComputationalPhysiology/simcardems/commit/5580bef
https://github.com/ComputationalPhysiology/simcardems/commit/bc47c96
https://github.com/ComputationalPhysiology/simcardems/commit/38ad52c
https://github.com/ComputationalPhysiology/simcardems/commit/7d2252c
https://github.com/ComputationalPhysiology/simcardems/commit/0648726

data = {}
for git_hash in all_hashes:
    results_file = benchmark_folder / git_hash / "results.json"
    if not results_file.is_file():
        print(f"Results does not exist for version {git_hash}")
        continue
    value = json.loads(results_file.read_text())
    if "mesh" in value.keys():
        # We have the wrong file
        continue
    data[git_hash] = value
        

results = defaultdict(list)

for git_hash, d in data.items():
    results["git_hash"].append(git_hash)
    for key, value in d.items():
        results[key].append(value)

df = pd.DataFrame(results)
df["timestamp"] = pd.to_datetime(df["timestamp"])
df
git_hash import_time ... rel_max_displacement_perc_z time_to_max_displacement_z
0 60e69df 9.000000e-07 ... 0.486886 0.05
1 cc84512 8.000000e-07 ... 0.486909 0.05
2 3398fd2 1.200000e-06 ... 0.486920 0.05
3 376167c 8.000000e-07 ... 0.486909 0.05
4 12ad5f2 9.000000e-07 ... 0.486909 0.05
... ... ... ... ... ...
87 5580bef 8.000000e-07 ... 0.486886 0.05
88 bc47c96 8.000000e-07 ... 0.486886 0.05
89 38ad52c 8.000000e-07 ... 0.523948 0.05
90 7d2252c 1.100000e-06 ... 0.486920 0.05
91 0648726 8.000000e-07 ... 0.249432 138.05

92 rows × 51 columns

df = df.sort_values(by='timestamp')
df
git_hash import_time ... rel_max_displacement_perc_z time_to_max_displacement_z
19 f63afc9 1.001000e-06 ... 0.249432 138.05
53 630ed09 1.000000e-06 ... 0.249432 138.05
68 4d3a653 1.100000e-06 ... 0.249432 138.05
75 0c195ff 1.000000e-06 ... 0.249432 138.05
80 c7df02b 9.000000e-07 ... 0.249432 138.05
... ... ... ... ... ...
65 5282d4f 1.200000e-06 ... 0.486886 0.05
45 1f7912f 9.000000e-07 ... 0.486886 0.05
14 813ba9c 1.200000e-06 ... 0.486886 0.05
82 04a9790 8.000000e-07 ... 0.486886 0.05
22 f4b9b88 8.000000e-07 ... 0.486886 0.05

92 rows × 51 columns

df[["simcardems_version", "timestamp", "git_hash"]]
simcardems_version timestamp git_hash
19 2022.3.0 2022-10-06 16:25:06.830650 f63afc9
53 2022.3.0 2022-10-06 16:55:04.711879 630ed09
68 2022.3.0 2022-10-06 20:09:07.230901 4d3a653
75 2022.3.0 2022-10-06 20:19:12.227464 0c195ff
80 2022.3.0 2022-10-06 20:28:25.015389 c7df02b
... ... ... ...
65 2023.2.0 2023-03-31 10:02:00.632466 5282d4f
45 2023.2.0 2023-03-31 13:39:54.344860 1f7912f
14 2023.3.0 2023-03-31 13:46:14.985304 813ba9c
82 2023.3.0 2023-04-04 17:25:25.501482 04a9790
22 2023.3.0 2023-04-11 08:24:24.930409 f4b9b88

92 rows × 3 columns

def get_ylim(values):
    if np.isclose(values, 0).all():
        return (-0.01, 0.01)
    y_mean = np.mean(values)
    y_max = max(np.max(values), y_mean + 0.01 * abs(y_mean))
    y_min = min(np.min(values), y_mean - 0.01 * abs(y_mean))
    return (y_min, y_max)

columns = [c for c in df.columns if c not in ["timestamp", "simcardems_version", "git_hash"]]
git_hash = df["git_hash"]
versions = df["simcardems_version"]
dates = [t.date() for t in df["timestamp"]]

text = []
for h,v, t in zip(git_hash, versions, dates):
    text.append("\n".join([
        f"<br>Git Hash: {h}</br>",
        f"<br>Version: {v}</br>",
        f"<br>Timestamp {t}</br>",
    ]))
    
fig = make_subplots(rows=len(columns), cols=1, subplot_titles=columns, shared_xaxes=True)

yranges = []
for i, c in enumerate(columns):
    
    row = i + 1
    col = 1
    
    fig.add_trace(
        go.Scatter(x=df["timestamp"], y=df[c], text=text, hovertemplate="%{text}"),
        row=row, col=col
    )
    fig.update_yaxes(range=get_ylim(df[c]),row=row, col=col)

fig.update_layout(
    height=5000,
    showlegend=False
)
fig.show()